{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from datetime import datetime\n",
    "from skimage import io\n",
    "from matplotlib import cm\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "lat_min, lat_max = 31.1, 31.4\n",
    "lon_min, lon_max = 121.3, 121.8\n",
    "lat_ctr = (int)((lat_max - lat_min) / 0.005)\n",
    "lon_ctr = (int)((lon_max - lon_min) / 0.005)\n",
    "\n",
    "def lat_quantize(lat):\n",
    "    return (int)((lat - lat_min) / 0.005)\n",
    "def lon_quantize(lon):\n",
    "    return (int)((lon - lon_min) / 0.005)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_hist(data):\n",
    "    hist, xedges, yedges = np.histogram2d(data['lat'], data['lon'],\n",
    "        bins = [range(lat_ctr),\n",
    "                range(lon_ctr)],\n",
    "        range=[[0, lat_ctr],\n",
    "               [0, lon_ctr]])\n",
    "    hist = hist.astype(\"int\")\n",
    "    return hist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_density_hour(data, hour_begin, hour_end):\n",
    "    data_hour = data[(hour_begin <= data['date_time']) & (data['date_time'] < hour_end)]\n",
    "    data_hour['lat'] = data_hour['lat'].apply(lambda lat:\n",
    "        (int)((lat-lat_min) / 0.005))\n",
    "    data_hour['lon'] = data_hour['lon'].apply(lambda lon:\n",
    "        (int)((lon-lon_min) / 0.005))\n",
    "    data_hour = data_hour.drop_duplicates(['car_id', 'lat', 'lon'])\n",
    "    hist  = extract_hist(data_hour)\n",
    "    return hist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_range_density(hist, range_xy):\n",
    "    center_lat = lat_quantize((range_xy[0][0] + range_xy[0][1]) / 2)\n",
    "    center_lon = lon_quantize((range_xy[1][0] + range_xy[1][1]) / 2)\n",
    "    return (int)(hist[center_lat][center_lon])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_str_range(string):\n",
    "    string_split = string.split('~')\n",
    "    return list(map(float, string_split))\n",
    "def get_submission_range():\n",
    "    X_sub = pd.read_csv(\"./A-test.csv\")\n",
    "    X_sub['latitude_range'] = X_sub['latitude_range'].apply(convert_str_range)\n",
    "    X_sub['longitude_range'] = X_sub['longitude_range'].apply(convert_str_range)\n",
    "    return X_sub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_dataset():\n",
    "    frames = []\n",
    "    features_need = ['car_id', 'date_time', 'lat', 'lon']\n",
    "    rcar_dtype = {'car_id': str,\n",
    "                  'date_time': str,\n",
    "                  'lat': float,\n",
    "                  'lon': float,\n",
    "                  'power_mode': str,\n",
    "                  'mileage': float,\n",
    "                  'speed': float,\n",
    "                  'fuel_consumption': float}\n",
    "    ecar_dtype = {'car_id': str,\n",
    "                  'date_time': str,\n",
    "                  'lat': float,\n",
    "                  'lon': float,\n",
    "                  'work_mode': str,\n",
    "                  'mileage': float,\n",
    "                  'speed': float,\n",
    "                  'avg_fuel_consumption': float,\n",
    "                  'system_mode': str}\n",
    "    for part in range(3):\n",
    "        filename = (\"./data/rcar/BOT_data_rcar_20170306_20170306_part{}.csv\"\n",
    "                    .format(part))\n",
    "        data_part = pd.read_csv(filename, dtype=rcar_dtype)[features_need]\n",
    "        data_part = data_part[(data_part['lat'] > 0.1) & (data_part['lon'] > 0.1)]\n",
    "        frames.append(data_part)\n",
    "        \n",
    "        filename = (\"./data/ecar/BOT_data_ecar_20170306_20170306_part{}.csv\"\n",
    "                    .format(part))\n",
    "        data_part = pd.read_csv(filename, dtype={'system_mode': str})[features_need]\n",
    "        data_part = data_part[(data_part['lat'] > 0.1) & (data_part['lon'] > 0.1)]\n",
    "        frames.append(data_part)\n",
    "        \n",
    "    data = pd.concat(frames)\n",
    "    data[\"date_time\"] = data[\"date_time\"].apply(lambda x: datetime.strptime(x, \"%Y-%m-%d %H:%M:%S\"))\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "rcar_dtype = {'car_id': str,\n",
    "                  'date_time': str,\n",
    "                  'lat': float,\n",
    "                  'lon': float,\n",
    "                  'power_mode': str,\n",
    "                  'mileage': float,\n",
    "                  'speed': float,\n",
    "                  'fuel_consumption': float}\n",
    "ecar_dtype = {'car_id': str,\n",
    "                  'date_time': str,\n",
    "                  'lat': float,\n",
    "                  'lon': float,\n",
    "                  'work_mode': str,\n",
    "                  'mileage': float,\n",
    "                  'speed': float,\n",
    "                  'avg_fuel_consumption': float,\n",
    "                  'system_mode': str}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv(\"./data/rcar/BOT_data_rcar_20170102_20170102_part0.csv\", dtype=rcar_dtype)\n",
    "data = data[(data['lat'] > 0.1) & (data['lon'] > 0.1)]\n",
    "data[\"date_time\"] = data[\"date_time\"].apply(lambda x: datetime.strptime(x, \"%Y-%m-%d %H:%M:%S\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jachinshen/.local/lib/python3.6/site-packages/ipykernel_launcher.py:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n",
      "/home/jachinshen/.local/lib/python3.6/site-packages/ipykernel_launcher.py:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \"\"\"\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAADnCAYAAAAO5q0KAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAEo5JREFUeJzt3XGMHOV9xvHniX3GsSMDjg/k2m4hwqKgSEBrYVOqiuJQUQcFS0ALSSurcnVSlaokTZU4+YtUjQRSFdJKFeop0PqPNAE5BFsIJQUHpFaKXOxCm4BD7boEXLvYbsGhSAWb/PrHDr695XZ3dnZ2dvad70dCt7M7u/Pe3PjHO8++844jQgCAyfeBcTcAAFAOCjoAJIKCDgCJoKADQCIo6ACQCAo6ACSCgg4AiRiqoNu+2fZLtg/b3lFWowAAg3PRC4tsL5L0b5JuknRU0rOS7oqIF8trHgAgr2F66NdKOhwRRyLiHUnfknRrOc0CAAxq8RDvXSPp1bblo5I29nrDEp8XS7V8iE1OnnfXn9f1tUWH3j73+Oyquf2y+NRbI20TUET7MdqJY3a03tTrpyJiut96wxR0L/Dc+/Ib2zOSZiRpqZZpozcPscnJc/ovL+v62vlbDp97fOq26849XjX7g5G2CSii/RjtxDE7Wk/Frp/kWW+Ygn5U0rq25bWSjnWuFBGzkmYlaYVXNnomsPYC3ol/EJg0HLP1M0yG/qyk9bYvtb1E0p2S9pTTLADAoAr30CPirO0/lPQ9SYskPRQRL5TWMgDAQIaJXBQRT0h6oqS2AACGMFRBx8JOzcx9eTSlk2NsCTCc9mMZ9cel/wCQCAo6ACSCyKUEvU5Lew1VBCYJwxTrjx46ACSCgg4AiaCgA0AiyNALas/ND9zzwLzXNj1/+9x6Gn7YF9klgDzooQNAIijoAJAIIpcS/PI9fzBveX5E0n3YYntsQ6yCOuDK0MlGDx0AEkFBB4BEVBq5nF21/NxdT1KKGFL6XYB2HNuThR46ACSCgg4AiaCgA0AiGj9ssegwramtbTeumC22rSKfARRR9Djv9T7y9fqhhw4AiaCgA0AiahO55D0lLOM0r4yr4c48Nt32edM91pzT2fb2ibvaP4NTWYxat2OMiGWy0UMHgERQ0AEgERR0AEjE2DL0QbK69nXLnqGw12fk3VbeTL7Xeu1DGDtvijGu7LLK7zVQviL/VvhbTra+PXTbD9k+YftHbc+ttP2k7UPZzwtH20wAQD95Ipe/lXRzx3M7JO2NiPWS9mbLAIAxckT0X8m+RNLjEfHRbPklSTdExHHbqyU9ExGX9/ucFV4ZG715uBaPWLeYYdTDJcuIfoBRHr8Yn6di14GI2NBvvaJfil4cEcclKft5UcHPAQCUZORfitqekTQjSUu1bNSbA4DGKlrQX7O9ui1yOdFtxYiYVTb11LLpdTFJN7ioYxs7T6nr2EZUhys70a5o5LJH0rbs8TZJu8tpDgCgqDzDFr8p6QeSLrd91PZ2SfdKusn2IUk3ZcsAgDHqG7lExF1dXqr3cBUAaJhaXinaKdUssMiMd73WrcsVpan+vSYB+77ZmMsFABJBQQeARFQauSw+9VauU8LOU/hRThJVxs0uRv35ZUwMNmnqECUBk4YeOgAkgoIOAImgoANAImpzk+h2eTPTvFn7IBnsKPPaUXx2XYYqYk7RfcN3BRgWPXQASAQFHQASUcvIJa+8w/mKXnmZd1ujNmlD+CatvaM0yO/PPVwxLHroAJAICjoAJGKiI5de2k9LB7kJQBmjN4b9jF5tqstEWEXjrqmtJ889PvPYdPkNG6FRR0lFR3cB76GHDgCJoKADQCIo6ACQiGQz9HZl5ORFs+smzpTY83fW3O/cnqefv+VwoW0V2Yej+N6kSgxbRDf00AEgERR0AEhEIyKXTmWfspY9GdMgwywnzbz2z849LON3LmPfFP2MSYttkCZ66ACQCAo6ACSCgg4AiXBEVLaxFV4ZG715wdfqkhszU+DkGudshRw3GKWnYteBiNjQb72+PXTb62w/bfug7Rds3509v9L2k7YPZT8vLKPhAIBi8kQuZyV9LiKukLRJ0qdtXylph6S9EbFe0t5sGQAwJn2HLUbEcUnHs8dv2j4oaY2kWyXdkK22U9Izkr4wikbW/XS2LjMgNsEgwwO7zbjJ3wupGuhLUduXSLpG0j5JF2fF/r2if1HZjQMA5Je7oNv+kKRvS/pMRPx0gPfN2N5ve/8ZvV2kjQCAHHJdKWp7Sq1i/o2IeDR7+jXbqyPiuO3Vkk4s9N6ImFV2TeAKr+w6pGZck1gVPf2uewyUqqL7utcNT+oywgoYVp5RLpb0oKSDEfHVtpf2SNqWPd4maXf5zQMA5JWnh369pN+V9EPbz2fPfUnSvZIesb1d0iuS7hhNEwEAeeQZ5fKPktzl5YWvEgIAVK6Rsy0WwVC3Zsr7XQnHA+qAuVwAIBEUdABIBJFLD9y0oDpVRlpNvM8rmoEeOgAkgoIOAImgoANAIsjQOxTJUJs+pLHopfN1zKub9rdDWuihA0AiKOgAkIiJiFx6zZTXzdTWk+cen3lseujtnn7isnmvnVL94oKyFY1EisQsRB3A8OihA0AiKOgAkIiJiFza5b4BhUZ7A4pekU6Rm19UecOMoqNyukVfRUeyELMA5aKHDgCJoKADQCIo6ACQiInL0Mfl/C2H5y33ypDbXxs2Tx/kfYN85rDKuFkzgHLRQweARFDQASARRC4l6BWRdItfqmhHN3mHGY4zIkI6qhyS23T00AEgERR0AEhEIyKXUUQCeScMKzve6PUZo3xP0c/nFLt56jjPfVPQQweARPQt6LaX2v4n2/9i+wXbX86ev9T2PtuHbD9se8nomwsA6CZPD/1tSTdGxFWSrpZ0s+1Nku6TdH9ErJf0uqTto2smAKCfvhl6RISk/80Wp7L/QtKNkj6ZPb9T0j2SHii/icUUuSlGGdvq3F7ZGXLn79Jt1scqs2tycvTC8VGdXBm67UW2n5d0QtKTkv5d0hsRcTZb5aikNaNpIgAgj1wFPSLejYirJa2VdK2kKxZabaH32p6xvd/2/jN6u3hLAQA9DTRsMSLesP2MpE2SLrC9OOulr5V0rMt7ZiXNStIKr1yw6FetjldDFo1pxhWzVGkShkGWcZUuMKw8o1ymbV+QPf6gpI9JOijpaUm3Z6ttk7R7VI0EAPSXp4e+WtJO24vU+h/AIxHxuO0XJX3L9p9Jek7SgyNsJwCgjzyjXP5V0jULPH9ErTwdAFADjbj0v9ewwiqVsd26/C7jMgkzO9axTVVq+u8/Tlz6DwCJoKADQCIaEbn0UuUpfNPjkpRMQvSD5qGHDgCJoKADQCLcmnurGiu8MjZ6c2XbyyPvqfMgp9jdopQi7+n3PgDpeyp2HYiIDf3Wo4cOAImgoANAIijoAJCIRg5bLDJccJAcu8jNNXq9ZxJmGwQwfvTQASARFHQASEQjI5d2VUYYeYc+ckXp+HAFKCYZPXQASAQFHQASQUEHgESMLUMvmlVO2hC+vMMRe5naenJuYbaUZgFIED10AEgEBR0AElHpbIvLptfF5bd9trLtddMeYZx5bHrea3WMcU4/cVmpn5f3dy46XLLK/VskgmNmS0waZlsEgIahoANAIiod5bL41Fu5TmmLnurnPuXWZF15ef6Ww0N/Rq99WmQCsV5Ob52LiOaN0NH8fV9GvFGXzwDqgB46ACQid0G3vcj2c7Yfz5Yvtb3P9iHbD9teMrpmAgD6GaSHfrekg23L90m6PyLWS3pd0vYyGwYAGEyuDN32Wkkfl/QVSX9s25JulPTJbJWdku6R9ECvzzm7arlO3dbKUHvllqPONIvcgGLURv29wbDvKer9+X++7wPKzvXLxqyMqKO8PfSvSfq8pJ9lyx+W9EZEnM2Wj0paU3LbAAAD6FvQbd8i6UREHGh/eoFVF7xCyfaM7f2295/9v7cKNhMA0E+eyOV6SZ+wvUXSUkkr1OqxX2B7cdZLXyvp2EJvjohZZVNKLZteV91lqSM2yqim1w0u6nhq37kvpnSyy5r5lXHV5yhN2t8IzdC3hx4RX4yItRFxiaQ7JX0/Ij4l6WlJt2erbZO0e2StBAD0Ncw49C+o9QXpYbUy9QfLaRIAoIiBrhSNiGckPZM9PiLp2vKbBAAoopaX/tdFlUPnUpoBsH2GxVMz013X47J9oFxc+g8AiaCgA0AixnZP0brqFn1UefXqqFUZJRGJANWhhw4AiaCgA0AiKo1c8k7ONU51nLirDL3297C/Z6+rJgFUhx46ACSCgg4AiaCgA0AixjZscRKujKxLO0atKb8nkDp66ACQCAo6ACSiNpNztUcwVQ57m9racTOG2co2nawmRDgMzUQd0UMHgERQ0AEgERR0AEhEbWZbHFfuenrrZfOXn7isy5rznb/l8CiagxqbhKG2aDZ66ACQCAo6ACSiNpHLuOSNTjqjmF7RDHFM+ohYUEf00AEgERR0AEiEI6Kyja3wytjozZVtryq94pczj013fY3TdgB5PBW7DkTEhn7r5crQbb8s6U1J70o6GxEbbK+U9LCkSyS9LOm3IuL1og0GAAxnkMjl1yPi6rb/S+yQtDci1kvamy0DAMZkmAz9Vkk7s8c7JW0dvjkAgKLyDlsMSX9vOyT9dUTMSro4Io5LUkQct33RqBpZR+25edGcvIwZ+8jhx6Pzb8ffAXWQt6BfHxHHsqL9pO0f592A7RlJM5K0VMsKNBEAkEeuyCUijmU/T0j6jqRrJb1me7UkZT9PdHnvbERsiIgNUzqvnFYDAN6nbw/d9nJJH4iIN7PHvyHpTyXtkbRN0r3Zz92jbOg49IpEpnSy62t5FTlN72wTk4kBeE+eyOViSd+x/d76fxcR37X9rKRHbG+X9IqkO0bXTABAP30LekQckXTVAs//t6T0rhICgAnFpf8AkIjGz7bYqT2jft8NpNu0D1Wscsha57ZO6boFXxtkdsh2ZO3dcWNo1B09dABIBAUdABJB5NJD71il3tHEINFJexyTN5opuq06KhKlcGUo6ogeOgAkgoIOAIkgckGhyKRoTFNlPEOUgqahhw4AiaCgA0AiKOgAkIjGZ+i9ctY65ql1uVoxbxbe2d5TM/musC3796zj3xIoGz10AEgEBR0AEpFU5NJ+ml70dH7STs3r3t6i7av77wXUET10AEgEBR0AEkFBB4BETHSG3nnJ+aotc7lr3mFvk5bVvu+mG7MLr9e5b9pnjuw0afsAwMLooQNAIijoAJAIR0R1G7NPSvqJpFWSTlW24XpjX8zH/pjDvpivyfvjFyKie26aqbSgn9uovT8iNlS+4RpiX8zH/pjDvpiP/dEfkQsAJIKCDgCJGFdB7zLYrpHYF/OxP+awL+Zjf/QxlgwdAFA+IhcASESlBd32zbZfsn3Y9o4qt10HttfZftr2Qdsv2L47e36l7SdtH8p+XjjutlbF9iLbz9l+PFu+1Pa+bF88bHvJuNtYFdsX2N5l+8fZMXJdU48N25/N/o38yPY3bS9t8rGRV2UF3fYiSX8l6TclXSnpLttXVrX9mjgr6XMRcYWkTZI+ne2DHZL2RsR6SXuz5aa4W9LBtuX7JN2f7YvXJW0fS6vG4y8kfTciflHSVWrtl8YdG7bXSPojSRsi4qOSFkm6U80+NnKpsod+raTDEXEkIt6R9C1Jt1a4/bGLiOMR8c/Z4zfV+ge7Rq39sDNbbaekreNpYbVsr5X0cUlfz5Yt6UZJu7JVmrQvVkj6NUkPSlJEvBMRb6ihx4Za80x90PZiScskHVdDj41BVFnQ10h6tW35aPZcI9m+RNI1kvZJujgijkutoi/povG1rFJfk/R5ST/Llj8s6Y2IOJstN+kY+Yikk5L+Jougvm57uRp4bETEf0r6c0mvqFXIT0s6oOYeG7lVWdC9wHONHGJj+0OSvi3pMxHx03G3Zxxs3yLpREQcaH96gVWbcowslvRLkh6IiGskvaUGxCsLyb4nuFXSpZJ+TtJytaLaTk05NnKrsqAflbSubXmtpGMVbr8WbE+pVcy/ERGPZk+/Znt19vpqSSfG1b4KXS/pE7ZfVit+u1GtHvsF2Wm21Kxj5KikoxGxL1vepVaBb+Kx8TFJ/xERJyPijKRHJf2Kmnts5FZlQX9W0vrsm+olan3JsafC7Y9dlhE/KOlgRHy17aU9krZlj7dJ2l1126oWEV+MiLURcYlax8L3I+JTkp6WdHu2WiP2hSRFxH9JetX25dlTmyW9qAYeG2pFLZtsL8v+zby3Lxp5bAyi6tkWt6jVC1sk6aGI+EplG68B278q6R8k/VBzufGX1MrRH5H082odzHdExP+MpZFjYPsGSX8SEbfY/ohaPfaVkp6T9DsR8fY421cV21er9QXxEklHJP2eWp2uxh0btr8s6bfVGhn2nKTfVyszb+SxkRdXigJAIrhSFAASQUEHgERQ0AEgERR0AEgEBR0AEkFBB4BEUNABIBEUdABIxP8D6l5Dy0P1FewAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda/lib/python3.6/site-packages/skimage/io/_io.py:140: UserWarning: ./fig/test.jpg is a low contrast image\n",
      "  warn('%s is a low contrast image' % fname)\n",
      "/opt/anaconda/lib/python3.6/site-packages/skimage/util/dtype.py:126: UserWarning: Possible sign loss when converting negative image of type int64 to positive image of type uint8.\n",
      "  .format(dtypeobj_in, dtypeobj_out))\n",
      "/opt/anaconda/lib/python3.6/site-packages/skimage/util/dtype.py:179: UserWarning: Downcasting int64 to uint8 without scaling because max value 3 fits in uint8\n",
      "  \"value {} fits in {}\".format(a.dtype, dtype, a.max(), dtype))\n"
     ]
    }
   ],
   "source": [
    "hist = get_density_hour(data, datetime(2017, 1, 2, 13), datetime(2017, 1, 2, 14))\n",
    "plt.imshow(hist)\n",
    "plt.show()\n",
    "io.imsave(\"./fig/test.jpg\", hist)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
